package Preprocess;


import org.apache.log4j.Logger;
import com.hankcs.hanlp.seg.common.Term;
import com.hankcs.hanlp.HanLP;
import java.io.*;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;

public class Step1_1_segment extends Thread{

    private static Queue<File> fileQueue;
    private static BufferedWriter outputWriter;


    private static Logger logger = Logger.getLogger(Step1_1_segment.class);
    private static String rootPath = "D:/data/thudata/";
    //private static String rootPath = "D:\\data\\thudata\\财经\\798977.txt";
    private static String outputFile = "D:/data/videodata_seg.txt";
    private static int cnt = 0;


    public static String readFile(File file) {
        Long fileLength = file.length();
        byte[] fileContent = new byte[fileLength.intValue()];
        try {
            FileInputStream in = new FileInputStream(file);
            in.read(fileContent);
            in.close();
        }catch (IOException e) {
            e.printStackTrace();
            logger.warn("文件读错误");
        }
        try {
            return new String(fileContent, "utf8");
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
            logger.warn("文件用utf8读错误");
            return null;
        }
    }

    public static void getAll(File root) {
        if (root == null) {
            logger.info("kong!");
            return;
        }
        if (root.isDirectory()) {
            File[] files = root.listFiles();
            if (files == null) return;
            for (File f : files) {
                getAll(f);
            }
        } else {
            fileQueue.add(root);
        }
    }

    static {
        fileQueue = new LinkedList<>();
        logger.info("读取所有文件...");
        getAll(new File(rootPath));
        logger.info("文件总共" + fileQueue.size());
        try {
            outputWriter = new BufferedWriter(new OutputStreamWriter(
                    new FileOutputStream(outputFile), "utf8"));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    @Override
    public void run() {
        while (true) {
            File curFile = null;
            synchronized(fileQueue) {
                curFile = fileQueue.poll();
            }
            if (curFile == null) {
                logger.info("没有文件了");
                break;
            }
            String content = readFile(curFile);
            if(content == null) {

            }
            List<Term> terms =HanLP.segment(content);
            StringBuilder line = new StringBuilder();
            for (Term t : terms) {
                line.append(t.word + ' ');
            }
            synchronized(outputWriter) {
                try {
                    outputWriter.write(line.toString() + '\n');
                    cnt++;
                    if (cnt % 1000 == 0)
                        logger.info(fileQueue.size() + " / " + cnt);
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }

    }


    public static void main(String[] args) throws IOException, InterruptedException {
        int nThread = 20;
        Step1_1_segment[] threads = new Step1_1_segment[nThread];

        for (int i = 0; i < nThread; i++) {
            threads[i] = new Step1_1_segment();
            threads[i].start();
            Thread.sleep(500);
        }
    }
}


/*
import pandas as pd
seg = pd.read_csv("", sep='\t', encoding='utf8', header=None)
seg = seg.drop_duplicates()
seg.to_csv(".", encoding='utf8', header=False, index=False)
*/